#=================
# Author: Chen Xin
# Date: 2017-09-19
#=================
import requests
import os
from datetime import datetime
import json
import gzip
import talib
import html5lib
import re
import StringIO
from pyquery import PyQuery as pq
import traceback

import numpy as np

proxies = {
    "http": "http://192.168.47.1:8580",
    "https": "http://192.168.47.1:8580",
}

def readdata(pagecount=1000):
    res=requests.get("https://stooq.com/q/d/l/?s=xauusd&i=d",proxies=proxies)
    if(res.status_code==200):
        data=[line.strip().split(",") for line in res.text.split("\n")]
        titles=data[0]
        data=data[1:]

        datalist=[]
        befortime=datetime(1980,1,1)
        for line in data:
            if len(line)<5:
                continue
            date=datetime.strptime(line[0],"%Y-%m-%d")
            if date<befortime:
                continue
            datalist.append({
                "Date":line[0],
                "Open":float(line[1]),
                "High":float(line[2]),
                "Low":float(line[3]),
                "Close":float(line[4]),
            })
        return datalist

def GetNew():
    def remove_control_characters(html):
        def str_to_int(s, default, base=10):
            if int(s, base) < 0x10000:
                return unichr(int(s, base))
            return default

        html = re.sub(ur"&#(\d+);?", lambda c: str_to_int(c.group(1), c.group(0)), html)
        html = re.sub(ur"&#[xX]([0-9a-fA-F]+);?", lambda c: str_to_int(c.group(1), c.group(0), base=16), html)
        html = re.sub(ur"[\x00-\x08\x0b\x0e-\x1f\x7f]", "", html)
        return html

    parser = html5lib.HTMLParser(tree=html5lib.treebuilders.getTreeBuilder("lxml"),namespaceHTMLElements=False)
    r = requests.get("https://stooq.com/q/?s=xauusd",proxies=proxies)
    try:
        try:
            doc = parser.parse(r.content)
        except:
            match = re.search(r"<meta.*content=\".*charset=(?P<code>[^\";]*)\".*/>", r.content)
            if match:
                code=match.group("code")
                b=r.content.decode(code,errors='ignore')
            b=remove_control_characters(b)
            doc=parser.parse(StringIO(b))
    except Exception,e:
        print e
        traceback.print_exc()
        return None
    doc=pq(doc.getroot())

    close = doc("#aq_xauusd_a2").text()
    high = doc("#aq_xauusd_h").text()
    low = doc("#aq_xauusd_l").text()
    open = doc("#aq_xauusd_o").text()
    date = doc("#aq_xauusd_d2").text()
    return {
        "Date":date,
        "Open":float(open),
        "High":float(high),
        "Low":float(low),
        "Close":float(close),
    }

def AddDI(datalist):
    for one in datalist:
        one["DI"]=(one["Close"]*2+one["Low"]+one["High"])/4
'''
def AddWMA(datalist):
    dis=np.array([one["DI"] for one in datalist])
    WMA = talib.MA(dis,30,matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["wma"]=WMA[i]
def AddMACD(datalist):
    dis=np.array([one["DI"] for one in datalist])
    macd, macdsignal, macdhist=talib.MACDEXT(dis, fastperiod=12, fastmatype=1, slowperiod=26, slowmatype=1, signalperiod=9, signalmatype=1)
    for i in xrange(len(datalist)):
        datalist[i]["macd"]=macd[i],macdsignal[i],macdhist[i]
def AddBBands(datalist):
    dis=np.array([one["DI"] for one in datalist])
    upperband, middleband, lowerband = talib.BBANDS(dis, timeperiod=15, nbdevup=2, nbdevdn=2, matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["bband"]=upperband[i], middleband[i], lowerband[i]
def AddRSI(datalist):
    dis=np.array([one["DI"] for one in datalist])
    res=talib.RSI(dis,14)
    for i in xrange(len(datalist)):
        datalist[i]["rsi"]=res[i]
def AddSTOCH(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    close=np.array([one["Close"] for one in datalist])
    slowk, slowd = talib.STOCH(high, low, close, fastk_period=5, slowk_period=3, slowk_matype=1, slowd_period=3, slowd_matype=1)
    for i in xrange(len(datalist)):
        datalist[i]["stoch"]=slowk[i], slowd[i]
def AddSAR(datalist):
    high=np.array([one["High"] for one in datalist])
    low=np.array([one["Low"] for one in datalist])
    optInAcceleration = .02
    optInMaximum = .20
    SAR = talib.SAR(high, low, optInAcceleration, optInMaximum)
    for i in xrange(len(datalist)):
        datalist[i]["sar"]=SAR[i]
'''

def savedata(data,dbfile):
    try:
        os.remove(dbfile)
    except:
        pass
    with gzip.GzipFile(dbfile,"w") as f:
        json.dump(data,f)
def loaddate(dbfile):
    with gzip.GzipFile(dbfile,"r") as f:
        return json.load(f)
if __name__ == "__main__":
    datalist=readdata()
    datalist.append(GetNew())
    #datalist=loaddate("data/golddata.gz")
    AddDI(datalist)
    AddWMA(datalist)
    AddMACD(datalist)
    AddBBands(datalist)
    AddRSI(datalist)
    AddSTOCH(datalist)
    AddSAR(datalist)
    print json.dumps(datalist,indent=2)
    savedata(datalist,"data/golddata.gz")
